{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download data from [kaggle](https://www.kaggle.com/shivinder/googlestockpricing/data)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"../datasets/Google.csv\")\n",
    "df.index = pd.DatetimeIndex(df['Date'].values)\n",
    "close = df[\"Close\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Open</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Close</th>\n",
       "      <th>Volume</th>\n",
       "      <th>Ex-Dividend</th>\n",
       "      <th>Split Ratio</th>\n",
       "      <th>Adj. Open</th>\n",
       "      <th>Adj. High</th>\n",
       "      <th>Adj. Low</th>\n",
       "      <th>Adj. Close</th>\n",
       "      <th>Adj. Volume</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2004-08-19</th>\n",
       "      <td>2004-08-19</td>\n",
       "      <td>100.01</td>\n",
       "      <td>104.06</td>\n",
       "      <td>95.96</td>\n",
       "      <td>100.335</td>\n",
       "      <td>44659000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>50.159839</td>\n",
       "      <td>52.191109</td>\n",
       "      <td>48.128568</td>\n",
       "      <td>50.322842</td>\n",
       "      <td>44659000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-20</th>\n",
       "      <td>2004-08-20</td>\n",
       "      <td>101.01</td>\n",
       "      <td>109.08</td>\n",
       "      <td>100.50</td>\n",
       "      <td>108.310</td>\n",
       "      <td>22834300.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>50.661387</td>\n",
       "      <td>54.708881</td>\n",
       "      <td>50.405597</td>\n",
       "      <td>54.322689</td>\n",
       "      <td>22834300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-23</th>\n",
       "      <td>2004-08-23</td>\n",
       "      <td>110.76</td>\n",
       "      <td>113.48</td>\n",
       "      <td>109.05</td>\n",
       "      <td>109.400</td>\n",
       "      <td>18256100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>55.551482</td>\n",
       "      <td>56.915693</td>\n",
       "      <td>54.693835</td>\n",
       "      <td>54.869377</td>\n",
       "      <td>18256100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-24</th>\n",
       "      <td>2004-08-24</td>\n",
       "      <td>111.24</td>\n",
       "      <td>111.60</td>\n",
       "      <td>103.57</td>\n",
       "      <td>104.870</td>\n",
       "      <td>15247300.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>55.792225</td>\n",
       "      <td>55.972783</td>\n",
       "      <td>51.945350</td>\n",
       "      <td>52.597363</td>\n",
       "      <td>15247300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-25</th>\n",
       "      <td>2004-08-25</td>\n",
       "      <td>104.76</td>\n",
       "      <td>108.00</td>\n",
       "      <td>103.88</td>\n",
       "      <td>106.000</td>\n",
       "      <td>9188600.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>52.542193</td>\n",
       "      <td>54.167209</td>\n",
       "      <td>52.100830</td>\n",
       "      <td>53.164113</td>\n",
       "      <td>9188600.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Date    Open    High     Low    Close      Volume  \\\n",
       "2004-08-19  2004-08-19  100.01  104.06   95.96  100.335  44659000.0   \n",
       "2004-08-20  2004-08-20  101.01  109.08  100.50  108.310  22834300.0   \n",
       "2004-08-23  2004-08-23  110.76  113.48  109.05  109.400  18256100.0   \n",
       "2004-08-24  2004-08-24  111.24  111.60  103.57  104.870  15247300.0   \n",
       "2004-08-25  2004-08-25  104.76  108.00  103.88  106.000   9188600.0   \n",
       "\n",
       "            Ex-Dividend  Split Ratio  Adj. Open  Adj. High   Adj. Low  \\\n",
       "2004-08-19          0.0          1.0  50.159839  52.191109  48.128568   \n",
       "2004-08-20          0.0          1.0  50.661387  54.708881  50.405597   \n",
       "2004-08-23          0.0          1.0  55.551482  56.915693  54.693835   \n",
       "2004-08-24          0.0          1.0  55.792225  55.972783  51.945350   \n",
       "2004-08-25          0.0          1.0  52.542193  54.167209  52.100830   \n",
       "\n",
       "            Adj. Close  Adj. Volume  \n",
       "2004-08-19   50.322842   44659000.0  \n",
       "2004-08-20   54.322689   22834300.0  \n",
       "2004-08-23   54.869377   18256100.0  \n",
       "2004-08-24   52.597363   15247300.0  \n",
       "2004-08-25   53.164113    9188600.0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "def get_daily_vol(close, span=100):\n",
    "    use_idx = close.index.searchsorted(close.index - pd.Timedelta(days=1))\n",
    "    use_idx = use_idx[use_idx > 0]\n",
    "    # Get rid of duplications in index\n",
    "    use_idx = np.unique(use_idx)\n",
    "    prev_idx = pd.Series(close.index[use_idx - 1], index=close.index[use_idx])\n",
    "    ret = close.loc[prev_idx.index] / close.loc[prev_idx.values].values - 1\n",
    "    vol = ret.ewm(span=span).std()\n",
    "    return vol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2004-08-23         NaN\n",
       "2004-08-24    0.036396\n",
       "2004-08-25    0.029930\n",
       "2004-08-26    0.027366\n",
       "2004-08-30    0.029365\n",
       "Name: Close, dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vol = get_daily_vol(df[\"Close\"])\n",
    "vol.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numbers\n",
    "\n",
    "\n",
    "def cusum_filter(close, h):\n",
    "    # asssum that E y_t = y_{t-1}\n",
    "    t_events = []\n",
    "    s_pos, s_neg = 0, 0\n",
    "    ret = close.pct_change().dropna()\n",
    "    diff = ret.diff().dropna()\n",
    "    # time variant threshold\n",
    "    if isinstance(h, numbers.Number):\n",
    "        h = pd.Series(h, index=diff.index)\n",
    "    h = h.reindex(diff.index, method='bfill')\n",
    "    h = h.dropna()\n",
    "    for t in h.index:\n",
    "        s_pos = max(0, s_pos + diff.loc[t])\n",
    "        s_neg = min(0, s_neg + diff.loc[t])\n",
    "        if s_pos > h.loc[t]:\n",
    "            s_pos = 0\n",
    "            t_events.append(t)\n",
    "        elif s_neg < -h.loc[t]:\n",
    "            s_neg = 0\n",
    "            t_events.append(t)\n",
    "    return pd.DatetimeIndex(t_events)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatetimeIndex(['2004-08-24', '2004-09-28', '2004-10-20', '2004-10-21',\n",
       "               '2004-10-26', '2004-11-05', '2004-11-08', '2004-11-11',\n",
       "               '2004-11-16', '2004-11-24', '2005-01-26', '2005-02-04',\n",
       "               '2005-06-08', '2005-10-21', '2005-10-25', '2006-01-20',\n",
       "               '2006-01-23', '2006-02-01', '2006-02-16', '2006-02-28',\n",
       "               '2006-03-02', '2006-03-24', '2006-04-25', '2006-10-20',\n",
       "               '2006-11-27', '2008-01-23', '2008-01-24', '2008-02-01',\n",
       "               '2008-02-05', '2008-03-11', '2008-03-17', '2008-03-24',\n",
       "               '2008-04-18', '2008-04-21', '2008-07-18', '2008-07-22',\n",
       "               '2008-09-11', '2008-09-17', '2008-09-18', '2008-09-22',\n",
       "               '2008-09-29', '2008-09-30', '2008-10-02', '2008-10-13',\n",
       "               '2008-10-14', '2008-10-16', '2008-10-28', '2008-10-29',\n",
       "               '2008-11-05', '2008-11-13', '2008-11-17', '2008-11-25',\n",
       "               '2008-12-01', '2008-12-02', '2008-12-22', '2009-01-20',\n",
       "               '2009-01-21', '2009-02-10', '2009-02-24', '2009-03-09',\n",
       "               '2009-03-10', '2010-01-22', '2010-04-16', '2010-05-10',\n",
       "               '2010-07-16', '2010-07-20', '2010-10-15', '2010-10-19',\n",
       "               '2011-04-15', '2011-05-17', '2011-07-15', '2011-07-18',\n",
       "               '2011-08-09', '2011-08-18', '2011-10-14', '2012-01-09',\n",
       "               '2012-01-27', '2012-10-18', '2012-11-19', '2013-10-18',\n",
       "               '2013-10-21', '2014-04-03', '2014-04-04', '2015-07-17',\n",
       "               '2015-07-20', '2015-08-26', '2015-09-01', '2016-04-22'],\n",
       "              dtype='datetime64[ns]', freq=None)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cusum_filter(df[\"Close\"], 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatetimeIndex(['2004-08-24', '2004-08-25', '2004-08-27', '2004-08-31',\n",
       "               '2004-09-02', '2004-09-03', '2004-09-07', '2004-09-10',\n",
       "               '2004-09-15', '2004-09-17',\n",
       "               ...\n",
       "               '2016-12-07', '2016-12-08', '2016-12-09', '2016-12-12',\n",
       "               '2016-12-15', '2016-12-30', '2017-01-03', '2017-01-04',\n",
       "               '2017-01-06', '2017-01-09'],\n",
       "              dtype='datetime64[ns]', length=1513, freq=None)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vol = get_daily_vol(close)\n",
    "sampled_idx = cusum_filter(close, vol)\n",
    "sampled_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1513,)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_idx.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3125, 13)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_t1(close, t_events, num_days):\n",
    "    t1 = close.index.searchsorted(t_events + pd.Timedelta(days=num_days))\n",
    "    t1 = t1[t1 < close.shape[0]]\n",
    "    t1 = pd.Series(close.index[t1], index=t_events[:t1.shape[0]])\n",
    "    return t1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1513,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2004-08-24   2004-08-25\n",
       "2004-08-25   2004-08-26\n",
       "2004-08-27   2004-08-30\n",
       "2004-08-31   2004-09-01\n",
       "2004-09-02   2004-09-03\n",
       "dtype: datetime64[ns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t1 = get_t1(close, sampled_idx, num_days=1)\n",
    "print(t1.shape)\n",
    "t1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def apply_ptslt1(close, events, ptsl, molecule):\n",
    "    \"\"\"Return datafram about if price touches the boundary\"\"\"\n",
    "    # Sample a subset with specific indices\n",
    "    _events = events.loc[molecule]\n",
    "    # Time limit\n",
    "    \n",
    "    out = pd.DataFrame(index=_events.index)\n",
    "    # Set Profit Taking and Stop Loss\n",
    "    if ptsl[0] > 0:\n",
    "        pt = ptsl[0] *  _events[\"trgt\"]\n",
    "    else:\n",
    "        # Switch off profit taking\n",
    "        pt = pd.Series(index=_events.index)\n",
    "    if ptsl[1] > 0:\n",
    "        sl = -ptsl[1] * _events[\"trgt\"]\n",
    "    else:\n",
    "        # Switch off stop loss\n",
    "        sl = pd.Series(index=_events.index)\n",
    "    # Replace undifined value with the last time index\n",
    "    time_limits = _events[\"t1\"].fillna(close.index[-1])\n",
    "    for loc, t1 in time_limits.iteritems():\n",
    "        df = close[loc:t1]\n",
    "        # Change the direction depending on the side\n",
    "        df = (df / close[loc] - 1) * _events.at[loc, 'side']\n",
    "        # print(df)\n",
    "        # print(loc, t1, df[df < sl[loc]].index.min(), df[df > pt[loc]].index.min())\n",
    "        out.at[loc, 'sl'] = df[df < sl[loc]].index.min()\n",
    "        out.at[loc, 'pt'] = df[df > pt[loc]].index.min()\n",
    "    out['t1'] = _events['t1'].copy(deep=True)\n",
    "    return out\n",
    "\n",
    "\n",
    "def get_3barriers(close, t_events, ptsl, trgt, min_ret=0, num_threads=1,\n",
    "                  t1=False, side=None):\n",
    "    # Get sampled target values\n",
    "    trgt = trgt.loc[t_events]\n",
    "    trgt = trgt[trgt > min_ret]\n",
    "    # Get time boundary t1\n",
    "    if t1 is False:\n",
    "        t1 = pd.Series(pd.NaT, index=t_events)\n",
    "    # Define the side\n",
    "    if side is None:\n",
    "        _side = pd.Series(1., index=trgt.index)\n",
    "        _ptsl = [ptsl, ptsl]\n",
    "    else:\n",
    "        _side = side.loc[trgt.index]\n",
    "        _ptsl = ptsl[:2]\n",
    "    events = pd.concat({'t1': t1, 'trgt': trgt, 'side': _side}, axis=1)\n",
    "    events = events.dropna(subset=['trgt'])\n",
    "    time_idx = apply_ptslt1(close, events, _ptsl, events.index)\n",
    "    # Skip when all of barrier are not touched\n",
    "    events['t1'] = time_idx.dropna(how='all').min(axis=1)\n",
    "    events = events.drop('side', axis=1)\n",
    "    return events"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tomoaki/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:36: FutureWarning: \n",
      "Passing list-likes to .loc or [] with any missing label will raise\n",
      "KeyError in the future, you can use .reindex() as an alternative.\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate-loc-reindex-listlike\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t1</th>\n",
       "      <th>trgt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2004-08-24</th>\n",
       "      <td>2004-08-25</td>\n",
       "      <td>0.036396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-25</th>\n",
       "      <td>2004-08-26</td>\n",
       "      <td>0.029930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-31</th>\n",
       "      <td>2004-09-01</td>\n",
       "      <td>0.026605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-09-02</th>\n",
       "      <td>2004-09-03</td>\n",
       "      <td>0.024097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-09-07</th>\n",
       "      <td>2004-09-08</td>\n",
       "      <td>0.023610</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   t1      trgt\n",
       "2004-08-24 2004-08-25  0.036396\n",
       "2004-08-25 2004-08-26  0.029930\n",
       "2004-08-31 2004-09-01  0.026605\n",
       "2004-09-02 2004-09-03  0.024097\n",
       "2004-09-07 2004-09-08  0.023610"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trgt = vol\n",
    "events = get_3barriers(close, t_events=sampled_idx, trgt=trgt,\n",
    "                       ptsl=1, t1=t1)\n",
    "events.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_bins(events, close):\n",
    "    # Prices algined with events\n",
    "    events = events.dropna(subset=['t1'])\n",
    "    px = events.index.union(events['t1'].values).drop_duplicates()\n",
    "    px = close.reindex(px, method='bfill')\n",
    "    # Create out object\n",
    "    out = pd.DataFrame(index=events.index)\n",
    "    out['ret'] = px.loc[events['t1'].values].values / px.loc[events.index] - 1.\n",
    "    if 'side' in events:\n",
    "        out['ret'] *= events['side']\n",
    "    out['bin'] = np.sign(out['ret'])\n",
    "    if 'side' in events:\n",
    "        out.loc[out['ret'] <= 0, 'bin'] = 0\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ret</th>\n",
       "      <th>bin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2004-08-24</th>\n",
       "      <td>0.010775</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-25</th>\n",
       "      <td>0.018019</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-31</th>\n",
       "      <td>-0.020709</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-09-02</th>\n",
       "      <td>-0.014777</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-09-07</th>\n",
       "      <td>0.007088</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ret  bin\n",
       "2004-08-24  0.010775  1.0\n",
       "2004-08-25  0.018019  1.0\n",
       "2004-08-31 -0.020709 -1.0\n",
       "2004-09-02 -0.014777 -1.0\n",
       "2004-09-07  0.007088  1.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins = get_bins(events, close)\n",
    "bins.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def drop_labels(events, min_pct=0.05):\n",
    "    while True:\n",
    "        df = events['bin'].value_counts(normalize=True)\n",
    "        if df.min() > min_pct or df.shape[0] < 3:\n",
    "            break\n",
    "        print('dropped label', df.argmin(), df.min())\n",
    "        events = events[events['bin'] != df.argmin()]\n",
    "    return events"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dropped label 0.0 0.0008216926869350862\n",
      "(1217, 2)\n",
      "(1216, 2)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tomoaki/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:6: FutureWarning: 'argmin' is deprecated. Use 'idxmin' instead. The behavior of 'argmin' will be corrected to return the positional minimum in the future. Use 'series.values.argmin' to get the position of the minimum now.\n",
      "  \n",
      "/home/tomoaki/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:7: FutureWarning: 'argmin' is deprecated. Use 'idxmin' instead. The behavior of 'argmin' will be corrected to return the positional minimum in the future. Use 'series.values.argmin' to get the position of the minimum now.\n",
      "  import sys\n"
     ]
    }
   ],
   "source": [
    "dropped_bins = drop_labels(bins)\n",
    "print(bins.shape)\n",
    "print(dropped_bins.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bins = dropped_bins"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_3barriers(close, t_events, ptsl, trgt, min_ret=0, num_threads=1,\n",
    "                  t1=False, side=None):\n",
    "    # Get sampled target values\n",
    "    trgt = trgt.loc[t_events]\n",
    "    trgt = trgt[trgt > min_ret]\n",
    "    # Get time boundary t1\n",
    "    if t1 is False:\n",
    "        t1 = pd.Series(pd.NaT, index=t_events)\n",
    "    # Define the side\n",
    "    if side is None:\n",
    "        _side = pd.Series(1., index=trgt.index)\n",
    "        _ptsl = [ptsl, ptsl]\n",
    "    else:\n",
    "        _side = side.loc[trgt.index]\n",
    "        _ptsl = ptsl[:2]\n",
    "    events = pd.concat({'t1': t1, 'trgt': trgt, 'side': _side}, axis=1)\n",
    "    events = events.dropna(subset=['trgt'])\n",
    "    time_idx = apply_ptslt1(close, events, _ptsl, events.index)\n",
    "    # Skip when all of barrier are not touched\n",
    "    time_idx = time_idx.dropna(how='all')\n",
    "    events['t1_type'] = time_idx.idxmin(axis=1)\n",
    "    events['t1'] = time_idx.min(axis=1)\n",
    "    if side is None:\n",
    "        events = events.drop('side', axis=1)\n",
    "    return events\n",
    "\n",
    "def get_bins(events, close):\n",
    "    # Prices algined with events\n",
    "    events = events.dropna(subset=['t1'])\n",
    "    px = events.index.union(events['t1'].values).drop_duplicates()\n",
    "    px = close.reindex(px, method='bfill')\n",
    "    # Create out object\n",
    "    out = pd.DataFrame(index=events.index)\n",
    "    out['ret'] = px.loc[events['t1'].values].values / px.loc[events.index] - 1.\n",
    "    if 'side' in events:\n",
    "        out['ret'] *= events['side']\n",
    "    out['bin'] = np.sign(out['ret'])\n",
    "    # 0 when touching vertical line\n",
    "    out['bin'].loc[events['t1_type'] == 't1'] = 0\n",
    "    if 'side' in events:\n",
    "        out.loc[out['ret'] <= 0, 'bin'] = 0\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tomoaki/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:4: FutureWarning: \n",
      "Passing list-likes to .loc or [] with any missing label will raise\n",
      "KeyError in the future, you can use .reindex() as an alternative.\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate-loc-reindex-listlike\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t1</th>\n",
       "      <th>trgt</th>\n",
       "      <th>t1_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2004-08-24</th>\n",
       "      <td>2004-08-25</td>\n",
       "      <td>0.036396</td>\n",
       "      <td>t1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-25</th>\n",
       "      <td>2004-08-26</td>\n",
       "      <td>0.029930</td>\n",
       "      <td>t1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-31</th>\n",
       "      <td>2004-09-01</td>\n",
       "      <td>0.026605</td>\n",
       "      <td>t1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-09-02</th>\n",
       "      <td>2004-09-03</td>\n",
       "      <td>0.024097</td>\n",
       "      <td>t1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-09-07</th>\n",
       "      <td>2004-09-08</td>\n",
       "      <td>0.023610</td>\n",
       "      <td>t1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   t1      trgt t1_type\n",
       "2004-08-24 2004-08-25  0.036396      t1\n",
       "2004-08-25 2004-08-26  0.029930      t1\n",
       "2004-08-31 2004-09-01  0.026605      t1\n",
       "2004-09-02 2004-09-03  0.024097      t1\n",
       "2004-09-07 2004-09-08  0.023610      t1"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t1 = get_t1(close, sampled_idx, num_days=1)\n",
    "events = get_3barriers(close, t_events=sampled_idx, trgt=trgt,\n",
    "                       ptsl=1, t1=t1)\n",
    "events.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['t1' 'pt' 'sl']\n",
      "count     1217\n",
      "unique       3\n",
      "top         t1\n",
      "freq       906\n",
      "Name: t1_type, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(events['t1_type'].unique())\n",
    "print(events['t1_type'].describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ret</th>\n",
       "      <th>bin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2004-08-24</th>\n",
       "      <td>0.010775</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-25</th>\n",
       "      <td>0.018019</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-31</th>\n",
       "      <td>-0.020709</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-09-02</th>\n",
       "      <td>-0.014777</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-09-07</th>\n",
       "      <td>0.007088</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ret  bin\n",
       "2004-08-24  0.010775  0.0\n",
       "2004-08-25  0.018019  0.0\n",
       "2004-08-31 -0.020709  0.0\n",
       "2004-09-02 -0.014777  0.0\n",
       "2004-09-07  0.007088  0.0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins = get_bins(events, close)\n",
    "bins.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0.0    906\n",
       " 1.0    180\n",
       "-1.0    131\n",
       "Name: bin, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins['bin'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import talib\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def macd_side(close):\n",
    "    macd, signal, hist = talib.MACD(close.values)\n",
    "    hist = pd.Series(hist).fillna(1).values\n",
    "    return pd.Series(2 * ((hist > 0).astype(float) - 0.5), index=close.index[-len(hist):])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tomoaki/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:4: FutureWarning: \n",
      "Passing list-likes to .loc or [] with any missing label will raise\n",
      "KeyError in the future, you can use .reindex() as an alternative.\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate-loc-reindex-listlike\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>side</th>\n",
       "      <th>t1</th>\n",
       "      <th>trgt</th>\n",
       "      <th>t1_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2004-08-24</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2004-08-25</td>\n",
       "      <td>0.036396</td>\n",
       "      <td>t1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-25</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2004-08-26</td>\n",
       "      <td>0.029930</td>\n",
       "      <td>t1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-31</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2004-09-01</td>\n",
       "      <td>0.026605</td>\n",
       "      <td>t1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-09-02</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2004-09-03</td>\n",
       "      <td>0.024097</td>\n",
       "      <td>t1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-09-07</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2004-09-08</td>\n",
       "      <td>0.023610</td>\n",
       "      <td>t1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            side         t1      trgt t1_type\n",
       "2004-08-24   1.0 2004-08-25  0.036396      t1\n",
       "2004-08-25   1.0 2004-08-26  0.029930      t1\n",
       "2004-08-31   1.0 2004-09-01  0.026605      t1\n",
       "2004-09-02   1.0 2004-09-03  0.024097      t1\n",
       "2004-09-07   1.0 2004-09-08  0.023610      t1"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "vol = get_daily_vol(close)\n",
    "sampled_idx = cusum_filter(close, vol)\n",
    "t1 = get_t1(close, sampled_idx, num_days=1)\n",
    "side =  macd_side(close)\n",
    "events = get_3barriers(close, t_events=sampled_idx, trgt=vol,\n",
    "                       ptsl=[1, 2], t1=t1, side=side)\n",
    "events.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ret</th>\n",
       "      <th>bin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2004-08-24</th>\n",
       "      <td>0.010775</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-25</th>\n",
       "      <td>0.018019</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-08-31</th>\n",
       "      <td>-0.020709</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-09-02</th>\n",
       "      <td>-0.014777</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004-09-07</th>\n",
       "      <td>0.007088</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ret  bin\n",
       "2004-08-24  0.010775  0.0\n",
       "2004-08-25  0.018019  0.0\n",
       "2004-08-31 -0.020709  0.0\n",
       "2004-09-02 -0.014777  0.0\n",
       "2004-09-07  0.007088  0.0"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins = get_bins(events, close)\n",
    "bins.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 1.])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins['bin'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "clf = RandomForestClassifier()\n",
    "x = np.hstack([events['side'].values[:, np.newaxis], close.loc[events.index].values[:, np.newaxis]])\n",
    "y = bins['bin'].values\n",
    "clf.fit(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., ..., 0., 0., 0.])"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.predict(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1217, 2)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.,  1.,  1., ..., -1., -1.,  1.])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "events['side'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on built-in function MACD in module talib.func:\n",
      "\n",
      "MACD(...)\n",
      "    MACD(real[, fastperiod=?, slowperiod=?, signalperiod=?])\n",
      "    \n",
      "    Moving Average Convergence/Divergence (Momentum Indicators)\n",
      "    \n",
      "    Inputs:\n",
      "        real: (any ndarray)\n",
      "    Parameters:\n",
      "        fastperiod: 12\n",
      "        slowperiod: 26\n",
      "        signalperiod: 9\n",
      "    Outputs:\n",
      "        macd\n",
      "        macdsignal\n",
      "        macdhist\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(talib.MACD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "macd, signal, hist = talib.MACD(close.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(macd[100:] - signal[100:]  - hist[100:] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3092,)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "macd[np.isfinite(macd)].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "signal = signal[np.isfinite(signal)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 1., 1., ..., 1., 1., 1.])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "2 * ((signal > 0).astype(float) - 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "macd.fill(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 1., 1., ..., 1., 1., 1.])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "macd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
